/**
 * @author Fang Lu, fanglupku@gmail.com
 * 2011-4-20
 */
package data.processor;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.neocrawler.entity.WebContent;

import com.exceptionhandler.service.crawl.ExceptionCrawler;

import util.FileDealer;

public class URLFilter {
	public static final String JAVA_ORG_FORUM_HOST = "http://www.java-forums.org";
	public static final String CODERANCH_HOST = "http://www.coderanch.com";
	public static final String ECLIPSE_FORUM_HOST = "http://www.eclipse.org/forums";
	public static final String JGRAPH_FORUM_HOST = "http://forum.jgraph.com";
	public static final String JAVA_NET_FORUM_HOST = "http://www.java.net";
	private static final Pattern PATTERN_JGRAPH_THREADURL  = Pattern.compile("/questions/\\d+/");
	private static final Pattern PATTERN_JGRAPH_SEEDURL = Pattern.compile("/questions/\\?sort=");
	public static void main(String args[]){
		try {
//			String content = FileDealer.getFileContent("C:/Users/ftc/Desktop/viewforum.php.htm");
//			ArrayList<String> urlList = getGoodURLList(content);
//			for (int i=0; i<urlList.size(); ++i){
//				System.out.println(i + ":" + urlList.get(i));
//				System.out.println(i + ":" + completeURL(urlList.get(i), "https://forum.hibernate.org/a/viewforum.php?f=1&sid=24b318f9d52b2084ff01de26804be456"));
//			}
			
			//test eclipse
			String content = FileDealer.getFileContent("C:\\Users\\ftc\\Desktop\\Exceptions\\forum\\Eclipse\\Eclipse Community Forums  M2M (model-to-model transformation).htm");
			ArrayList<WebContent> urlList = getWebContentList_EclipseForum(content);
			for (int i=0; i<urlList.size(); ++i){
//				if (urlList.get(i).getType().equals(WebContent.TYPE_SEED_PAGE))
				System.out.println(i + "/" + urlList.get(i).getType() + ":" + urlList.get(i).getUrl());
			}
			
			//test java-forum
//			String content = FileDealer.getFileContent("C:\\Users\\ftc\\Desktop\\Exceptions\\forum\\java-forums\\Java Applets - Java Forums.htm");
//			ArrayList<WebContent> urlList = getWebContentList_JAVAFORUM(content);
//			for (int i=0; i<urlList.size(); ++i){
//				if (urlList.get(i).getType().equals(WebContent.TYPE_SEED_PAGE))
//					System.out.println(i + "/" + urlList.get(i).getType() + ":" + urlList.get(i).getUrl());
//			}
			
			//test coderanch
//			String content = FileDealer.getFileContent("C:\\Users\\ftc\\Desktop\\Exceptions\\forum\\coderanch\\sockets.htm");
//			ArrayList<WebContent> urlList = getWebContentList_CODERANCH(content);
//			for (int i=0; i<urlList.size(); ++i){
//				if (urlList.get(i).getType().equals(WebContent.TYPE_SEED_PAGE))
//				System.out.println(i + "/" + urlList.get(i).getType() + ":" + urlList.get(i).getUrl());
//			}
			
			//test jgraph
//			String content = FileDealer.getFileContent("C:\\Users\\ftc\\Desktop\\Exceptions\\forum\\jgraph\\All Questions - JGraph.htm");
//			ArrayList<WebContent> urlList = getWebContentList_JGRAPHFORUM(content);
//			for (int i=0; i<urlList.size(); ++i){
////				if (urlList.get(i).getType().equals(WebContent.TYPE_SEED_PAGE))
//				System.out.println(i + "/" + urlList.get(i).getType() + ":" + urlList.get(i).getUrl());
//			}
			
			//test java-net
//			String content = FileDealer.getFileContent("C:\\Users\\ftc\\Desktop\\Exceptions\\forum\\java-net\\java-development-tools.htm");
//			ArrayList<WebContent> urlList = getWebContentList_JAVANET(content);
//			for (int i=0; i<urlList.size(); ++i){
////				if (urlList.get(i).getType().equals(WebContent.TYPE_SEED_PAGE))
//				System.out.println(i + "/" + urlList.get(i).getType() + ":" + urlList.get(i).getUrl());
//			}
			
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	public static ArrayList<WebContent> filterByHost(ArrayList<WebContent> srcList, String hostString){
		ArrayList<WebContent> resultList = new ArrayList<WebContent>();
		for (WebContent webContent : srcList) {
			if (webContent.getUrl().startsWith(hostString + "/") 
					|| webContent.getUrl().equals(hostString)){
				resultList.add(webContent);
			}
		}
		return resultList;
	}
	/**
	 * default is hibernate
	 * @param content
	 * @return
	 */
	public static ArrayList<String> getGoodURLList(String content){
		ArrayList<String> goodURLList = new ArrayList<String>();
		String strPattern = "<a\\s+.*?href=\"[^\"]*\"[^>]*>";
		Pattern pattern = Pattern.compile(strPattern, 
				Pattern.CASE_INSENSITIVE| Pattern.DOTALL);
		Matcher matcher = pattern.matcher(content);
		while (matcher.find()){
			String thisAnchor = new String(content.substring(matcher.start(), matcher.end()));
			int begin_pos = thisAnchor.indexOf("href=\"");
			if (begin_pos >= 0){
				int end_pos = thisAnchor.indexOf("\"", begin_pos + 6);
				if (end_pos >= 0){
					String thisGoodURL = new String( thisAnchor.substring(begin_pos+6, end_pos));
					if (thisGoodURL.indexOf("viewforum.php") >= 0
							|| thisGoodURL.indexOf("viewtopic.php") >= 0){
						goodURLList.add(thisGoodURL);
					}
				}
			}
		}
		
		return goodURLList;
	}
	public static ArrayList<WebContent> getWebContentList_JAVAFORUM(String content){
		ArrayList<WebContent> webContentList = new ArrayList<WebContent>();
		HashSet<String> urlSet = new HashSet<String>(); 
		String strPattern = "<a\\s+.*?href=\"[^\"]*\"[^>]*>";
		Pattern pattern = Pattern.compile(strPattern, 
				Pattern.CASE_INSENSITIVE| Pattern.DOTALL);
		Matcher matcher = pattern.matcher(content);
		while (matcher.find()){
			String thisAnchor = new String(content.substring(matcher.start(), matcher.end()));
			int begin_pos = thisAnchor.indexOf("href=\"");
			if (begin_pos >= 0){
				int end_pos = thisAnchor.indexOf("\"", begin_pos + 6);
				if (end_pos >= 0){
					String thisGoodURL = new String( thisAnchor.substring(begin_pos+6, end_pos));
					if (thisGoodURL.toLowerCase().indexOf("http://www.java-forums.org/") >= 0
							&& thisGoodURL.toLowerCase().endsWith(".html")){
						if (urlSet.add(thisGoodURL)){
							boolean isSeedPage = true;
							try{
								int tempPos = thisGoodURL.lastIndexOf("/") + 1;
								String thisPageName = new String(thisGoodURL.substring(tempPos, thisGoodURL.length() - 5));
								if (!thisPageName.startsWith("index")){
									isSeedPage = false;
								}
								tempPos = 5;
								while (isSeedPage && tempPos < thisPageName.length()){
									if (thisPageName.charAt(tempPos) > '9' || thisPageName.charAt(tempPos) < '0'){
										isSeedPage = false;
										break;
									}
									++ tempPos;
								}
							}catch (Exception e) {
								// TODO: handle exception
							}
							WebContent thisWebContent = new WebContent();
							thisWebContent.setUrl(thisGoodURL);
							if (isSeedPage){
								thisWebContent.setType(WebContent.TYPE_SEED_PAGE);
							}else{
								thisWebContent.setType(ExceptionCrawler.TYPE_FORUM_THREAD);
							}
							webContentList.add(thisWebContent);
						}
						
					}else if (thisGoodURL.toLowerCase().indexOf("http://www.java-forums.org/") >= 0
							&& thisGoodURL.toLowerCase().endsWith("/")){
						if (urlSet.add(thisGoodURL)){
							WebContent thisWebContent = new WebContent();
							thisWebContent.setUrl(thisGoodURL);
							thisWebContent.setType(WebContent.TYPE_SEED_PAGE);
							webContentList.add(thisWebContent);
						}
					}
				}
			}
		}
		return filterByHost(webContentList, JAVA_ORG_FORUM_HOST);
	}
	public static ArrayList<WebContent> getWebContentList_CODERANCH(String content){
		ArrayList<WebContent> webContentList = new ArrayList<WebContent>();
		HashSet<String> urlSet = new HashSet<String>(); 
		String strPattern = "<a\\s+.*?href=\"[^\"]*\"[^>]*>";
		Pattern pattern = Pattern.compile(strPattern, 
				Pattern.CASE_INSENSITIVE| Pattern.DOTALL);
		Matcher matcher = pattern.matcher(content);
		while (matcher.find()){
			String thisAnchor = new String(content.substring(matcher.start(), matcher.end()));
			int begin_pos = thisAnchor.indexOf("href=\"");
			if (begin_pos >= 0){
				int end_pos = thisAnchor.indexOf("\"", begin_pos + 6);
				if (end_pos >= 0){
					String thisGoodURL = new String( thisAnchor.substring(begin_pos+6, end_pos));
					if (thisGoodURL.toLowerCase().startsWith("/t/")){
						thisGoodURL = CODERANCH_HOST + thisGoodURL;
						if (urlSet.add(thisGoodURL)){
							WebContent thisWebContent = new WebContent();
							thisWebContent.setUrl(thisGoodURL);
							thisWebContent.setType(ExceptionCrawler.TYPE_FORUM_THREAD);
							webContentList.add(thisWebContent);
						}
						
					}else if (thisGoodURL.toLowerCase().startsWith("/forums/f-")||
							thisGoodURL.toLowerCase().startsWith("/forums/forums/show/")){
						thisGoodURL = CODERANCH_HOST + thisGoodURL;
						if (urlSet.add(thisGoodURL)){
							WebContent thisWebContent = new WebContent();
							thisWebContent.setUrl(thisGoodURL);
							thisWebContent.setType(WebContent.TYPE_SEED_PAGE);
							webContentList.add(thisWebContent);
						}
					}
				}
			}
		}
		return filterByHost(webContentList, CODERANCH_HOST);
	}
	public static ArrayList<WebContent> getWebContentList_EclipseForum(String content){
		ArrayList<WebContent> webContentList = new ArrayList<WebContent>();
		HashSet<String> urlSet = new HashSet<String>(); 
		String strPattern = "<a\\s+.*?href=\"[^\"]*\"[^>]*>";
		Pattern pattern = Pattern.compile(strPattern, 
				Pattern.CASE_INSENSITIVE| Pattern.DOTALL);
		Matcher matcher = pattern.matcher(content);
		while (matcher.find()){
			String thisAnchor = new String(content.substring(matcher.start(), matcher.end()));
			int begin_pos = thisAnchor.indexOf("href=\"");
			if (begin_pos >= 0){
				int end_pos = thisAnchor.indexOf("\"", begin_pos + 6);
				if (end_pos >= 0){
					String thisGoodURL = new String( thisAnchor.substring(begin_pos+6, end_pos));
					if (thisGoodURL.toLowerCase().indexOf("index.php/t/") >= 0 
							|| thisGoodURL.toLowerCase().indexOf("index.php/m/") >= 0){
						if (!thisGoodURL.startsWith(ECLIPSE_FORUM_HOST)){
							if (thisGoodURL.startsWith("/")){
								thisGoodURL = ECLIPSE_FORUM_HOST + thisGoodURL;
							}else{
								thisGoodURL = ECLIPSE_FORUM_HOST + "/" + thisGoodURL;
							}
						}
						if (urlSet.add(thisGoodURL)){
							WebContent thisWebContent = new WebContent();
							thisWebContent.setUrl(thisGoodURL);
							thisWebContent.setType(ExceptionCrawler.TYPE_FORUM_THREAD);
							webContentList.add(thisWebContent);
						}
						
					}else if (thisGoodURL.toLowerCase().indexOf("index.php/f/") >= 0
							|| thisGoodURL.toLowerCase().indexOf("index.php/sf/thread/") >= 0){
						if (!thisGoodURL.startsWith(ECLIPSE_FORUM_HOST)){
							if (thisGoodURL.startsWith("/")){
								thisGoodURL = ECLIPSE_FORUM_HOST + thisGoodURL;
							}else{
								thisGoodURL = ECLIPSE_FORUM_HOST + "/" + thisGoodURL;
							}
						}
						if (urlSet.add(thisGoodURL)){
							WebContent thisWebContent = new WebContent();
							thisWebContent.setUrl(thisGoodURL);
							thisWebContent.setType(WebContent.TYPE_SEED_PAGE);
							webContentList.add(thisWebContent);
						}
					}
				}
			}
		}
		return filterByHost(webContentList, ECLIPSE_FORUM_HOST);
	}
	public static ArrayList<WebContent> getWebContentList_JGRAPHFORUM(String content){
		ArrayList<WebContent> webContentList = new ArrayList<WebContent>();
		HashSet<String> urlSet = new HashSet<String>(); 
		String strPattern = "<a\\s+.*?href=\"[^\"]*\"[^>]*>";
		Pattern pattern = Pattern.compile(strPattern, 
				Pattern.CASE_INSENSITIVE| Pattern.DOTALL);
		Matcher matcher = pattern.matcher(content);
		while (matcher.find()){
			String thisAnchor = new String(content.substring(matcher.start(), matcher.end()));
			int begin_pos = thisAnchor.indexOf("href=\"");
			if (begin_pos >= 0){
				int end_pos = thisAnchor.indexOf("\"", begin_pos + 6);
				if (end_pos >= 0){
					String thisGoodURL = new String( thisAnchor.substring(begin_pos+6, end_pos));
//					System.out.println(thisGoodURL);
					Matcher thisThreadMatcher = PATTERN_JGRAPH_THREADURL.matcher(thisGoodURL);
					if (thisThreadMatcher.find()){
						if (thisGoodURL.startsWith(JGRAPH_FORUM_HOST)){	
						}else{
							thisGoodURL = JGRAPH_FORUM_HOST + thisGoodURL;
						}
						if (urlSet.add(thisGoodURL)){
							WebContent thisWebContent = new WebContent();
							thisWebContent.setUrl(thisGoodURL);
							thisWebContent.setType(ExceptionCrawler.TYPE_FORUM_THREAD);
							webContentList.add(thisWebContent);
						}
					}
					Matcher thisSeedMatcher = PATTERN_JGRAPH_SEEDURL.matcher(thisGoodURL);
					if (thisSeedMatcher.find()){
						if (thisGoodURL.startsWith(JGRAPH_FORUM_HOST)){	
						}else{
							thisGoodURL = JGRAPH_FORUM_HOST + thisGoodURL;
						}
						if (urlSet.add(thisGoodURL)){
							WebContent thisWebContent = new WebContent();
							thisWebContent.setUrl(thisGoodURL);
							thisWebContent.setType(WebContent.TYPE_SEED_PAGE);
							webContentList.add(thisWebContent);
						}
					}
				}
			}
		}
		return filterByHost(webContentList, JGRAPH_FORUM_HOST);
	}
	public static ArrayList<WebContent> getWebContentList_JAVANET(String content){
		ArrayList<WebContent> webContentList = new ArrayList<WebContent>();
		HashSet<String> urlSet = new HashSet<String>(); 
		String strPattern = "<a\\s+.*?href=\"[^\"]*\"[^>]*>";
		Pattern pattern = Pattern.compile(strPattern, 
				Pattern.CASE_INSENSITIVE| Pattern.DOTALL);
		Matcher matcher = pattern.matcher(content);
		while (matcher.find()){
			String thisAnchor = new String(content.substring(matcher.start(), matcher.end()));
			int begin_pos = thisAnchor.indexOf("href=\"");
			if (begin_pos >= 0){
				int end_pos = thisAnchor.indexOf("\"", begin_pos + 6);
				if (end_pos >= 0){
					String thisGoodURL = new String( thisAnchor.substring(begin_pos+6, end_pos));
//					System.out.println(thisGoodURL);
					if (thisGoodURL.indexOf("/forum/topic/") >= 0){
						if (thisGoodURL.startsWith(JAVA_NET_FORUM_HOST)){
						}else{
							thisGoodURL = JAVA_NET_FORUM_HOST + thisGoodURL;
						}
						
						if (urlSet.add(thisGoodURL)){
							WebContent thisWebContent = new WebContent();
							thisWebContent.setUrl(thisGoodURL);
							thisWebContent.setType(ExceptionCrawler.TYPE_FORUM_THREAD);
							webContentList.add(thisWebContent);
						}
					}
					else if(thisGoodURL.indexOf("/forums/") >= 0){ 
						if (thisGoodURL.startsWith(JAVA_NET_FORUM_HOST)){
						}else{
							thisGoodURL = JAVA_NET_FORUM_HOST + thisGoodURL;
						}
						if (urlSet.add(thisGoodURL)){
							WebContent thisWebContent = new WebContent();
							thisWebContent.setUrl(thisGoodURL);
							thisWebContent.setType(WebContent.TYPE_SEED_PAGE);
							webContentList.add(thisWebContent);
						}
					}
				}
			}
		}
		return filterByHost(webContentList, JAVA_NET_FORUM_HOST);
	}
//	public static ArrayList<String> getGoodURLList_JAVAFORUM(String content){
//		ArrayList<String> goodURLList = new ArrayList<String>();
//		String strPattern = "<a\\s+.*?href=\"[^\"]*\"[^>]*>";
//		Pattern pattern = Pattern.compile(strPattern, 
//				Pattern.CASE_INSENSITIVE| Pattern.DOTALL);
//		Matcher matcher = pattern.matcher(content);
//		while (matcher.find()){
//			String thisAnchor = new String(content.substring(matcher.start(), matcher.end()));
//			int begin_pos = thisAnchor.indexOf("href=\"");
//			if (begin_pos >= 0){
//				int end_pos = thisAnchor.indexOf("\"", begin_pos + 6);
//				if (end_pos >= 0){
//					String thisGoodURL = new String( thisAnchor.substring(begin_pos+6, end_pos));
//					if (thisGoodURL.toLowerCase().indexOf("http://www.java-forums.org") >= 0){
//						goodURLList.add(thisGoodURL);
//					}
//				}
//			}
//		}
//		return goodURLList;
//	}
	
	public static boolean isURLNeedComplete(String originURL){
		try{
			new URL(originURL);
		}catch (Exception e) {
			return true;
		}
		return false;
	}
	
//	public static String getCurrentPath(String sourceURL){
//		int pPos = sourceURL.indexOf("//");
//		if (pPos > 0){
//			
//		}
//	}
	
	public static String completeURL(String targetURL, String sourceURL){
		if (isURLNeedComplete(targetURL)){
			URL url;
			try {
				url = new URL(sourceURL);
				if (targetURL.trim().startsWith("./")){
//					System.out.println("file:" + url.getFile());
//					System.out.println("path:" + url.getPath());
					String dir = url.getPath();
					int dirPos = url.getPath().lastIndexOf("/");
					if (dirPos < 0){
						dir = "/";
					}else{
						dir = new String(dir.substring(0, dirPos+1));
					}
					if (!dir.endsWith("/")){
						dir += "/";
					}
					targetURL = targetURL.replace("./", url.getProtocol() + "://" + url.getHost() + dir);
				}else if (targetURL.trim().startsWith("/")){
					targetURL = targetURL.replace("/", url.getProtocol() + "://" + url.getHost() + "/");
				}else{
					targetURL = url.getProtocol() + "://" + url.getHost() + "/" + url.getPath() + "/" + targetURL;
				}
			} catch (MalformedURLException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}else{
		}
		return targetURL;
	}
}
